{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'\\n本代码实现基于分类后的任务数据.csv来进行针对该任务的文本分类模型训练\\n目的：根据已有的任务与其对应的类型，我们训练出可以预测新任务类型的模型。\\n实现的步骤：\\n\\n1.数据处理\\n    根据类别分布空间比例拆分训练集与测试集\\n\\n2.搭建词向量矩阵\\n'"
      ]
     },
     "metadata": {},
     "execution_count": 1
    }
   ],
   "source": [
    "\"\"\"\n",
    "本代码实现基于分类后的任务数据.csv来进行针对该任务的文本分类模型训练\n",
    "目的：根据已有的任务与其对应的类型，我们训练出可以预测新任务类型的模型。\n",
    "实现的步骤：\n",
    "\n",
    "1.数据处理\n",
    "    根据类别分布空间比例拆分训练集与测试集\n",
    "\n",
    "2.搭建词向量矩阵\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pathlib\n",
    "import pandas as pd\n",
    "import jieba\n",
    "import numpy as np \n",
    "import tensorflow as tf \n",
    "from gensim.models import KeyedVectors\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "' 数据查看 '"
      ]
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "\"\"\" 数据查看 \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   Unnamed: 0  Unnamed: 0.1                          task  \\\n",
       "0           0          8447        宣传片拍摄纪录片录制创意故事企业定制剪辑特效   \n",
       "1           1          2053                      商业MG动画制作   \n",
       "2           2          2514   深圳广州MG创意动画flash各类二维动画宣传手绘动画   \n",
       "3           3          7273  专业男声女声录音童话故事配音童声动画配音外语英语配音制作   \n",
       "4           4          2959                       科普宣传演示类   \n",
       "\n",
       "                                  clip_task  cluster_type  index  Name  \\\n",
       "0           宣传片 拍摄 纪录片 录制 创意 故事 企业 定制 剪辑 特效             0      0   叶孤城   \n",
       "1                               商业 MG 动画 制作             0      1  西门吹雪   \n",
       "2    深圳 广州 MG 创意 动画 flash 各类 二维 动画 宣传 手绘 动画             0      2   花满楼   \n",
       "3  专业 男声 女声 录音 童话故事 配音 童声 动画 配音 外语 英语 配音 制作             8      3  司空摘星   \n",
       "4                                科普 宣传 演示 类             0      4   楚留香   \n",
       "\n",
       "   Gender  Age  Years  Age_id  Name_id  \n",
       "0       1   24      4       2        0  \n",
       "1       1   25      5       2        1  \n",
       "2       1   20      2       2        2  \n",
       "3       1   30      4       3        3  \n",
       "4       1   24      3       2        4  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Unnamed: 0</th>\n      <th>Unnamed: 0.1</th>\n      <th>task</th>\n      <th>clip_task</th>\n      <th>cluster_type</th>\n      <th>index</th>\n      <th>Name</th>\n      <th>Gender</th>\n      <th>Age</th>\n      <th>Years</th>\n      <th>Age_id</th>\n      <th>Name_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>8447</td>\n      <td>宣传片拍摄纪录片录制创意故事企业定制剪辑特效</td>\n      <td>宣传片 拍摄 纪录片 录制 创意 故事 企业 定制 剪辑 特效</td>\n      <td>0</td>\n      <td>0</td>\n      <td>叶孤城</td>\n      <td>1</td>\n      <td>24</td>\n      <td>4</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>2053</td>\n      <td>商业MG动画制作</td>\n      <td>商业 MG 动画 制作</td>\n      <td>0</td>\n      <td>1</td>\n      <td>西门吹雪</td>\n      <td>1</td>\n      <td>25</td>\n      <td>5</td>\n      <td>2</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>2514</td>\n      <td>深圳广州MG创意动画flash各类二维动画宣传手绘动画</td>\n      <td>深圳 广州 MG 创意 动画 flash 各类 二维 动画 宣传 手绘 动画</td>\n      <td>0</td>\n      <td>2</td>\n      <td>花满楼</td>\n      <td>1</td>\n      <td>20</td>\n      <td>2</td>\n      <td>2</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>7273</td>\n      <td>专业男声女声录音童话故事配音童声动画配音外语英语配音制作</td>\n      <td>专业 男声 女声 录音 童话故事 配音 童声 动画 配音 外语 英语 配音 制作</td>\n      <td>8</td>\n      <td>3</td>\n      <td>司空摘星</td>\n      <td>1</td>\n      <td>30</td>\n      <td>4</td>\n      <td>3</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>2959</td>\n      <td>科普宣传演示类</td>\n      <td>科普 宣传 演示 类</td>\n      <td>0</td>\n      <td>4</td>\n      <td>楚留香</td>\n      <td>1</td>\n      <td>24</td>\n      <td>3</td>\n      <td>2</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "current_dir=pathlib.Path()\n",
    "dataset_dir =pathlib.Path(current_dir,'Dataset')\n",
    "data_csv=pathlib.Path(dataset_dir,'添加了制造者信息的任务数据.csv')\n",
    "data=pd.read_csv(data_csv)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "       Unnamed: 0  Unnamed: 0.1 task clip_task  cluster_type  index Name  \\\n",
       "1050         1050          3582  NaN       NaN             0   1050  陆小凤   \n",
       "4557         4557         11737  NaN       NaN             0   4557  丁灵琳   \n",
       "12331       12331         11676  NaN       NaN             0  12331  燕十三   \n",
       "\n",
       "       Gender  Age  Years  Age_id  Name_id  \n",
       "1050        1   26      8       3       10  \n",
       "4557        0   18      1       1       13  \n",
       "12331       1   25      4       2       11  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Unnamed: 0</th>\n      <th>Unnamed: 0.1</th>\n      <th>task</th>\n      <th>clip_task</th>\n      <th>cluster_type</th>\n      <th>index</th>\n      <th>Name</th>\n      <th>Gender</th>\n      <th>Age</th>\n      <th>Years</th>\n      <th>Age_id</th>\n      <th>Name_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1050</th>\n      <td>1050</td>\n      <td>3582</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>1050</td>\n      <td>陆小凤</td>\n      <td>1</td>\n      <td>26</td>\n      <td>8</td>\n      <td>3</td>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>4557</th>\n      <td>4557</td>\n      <td>11737</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>4557</td>\n      <td>丁灵琳</td>\n      <td>0</td>\n      <td>18</td>\n      <td>1</td>\n      <td>1</td>\n      <td>13</td>\n    </tr>\n    <tr>\n      <th>12331</th>\n      <td>12331</td>\n      <td>11676</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>12331</td>\n      <td>燕十三</td>\n      <td>1</td>\n      <td>25</td>\n      <td>4</td>\n      <td>2</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "#找出有空值的行，删除掉\n",
    "data[data.isnull().T.any()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=data.drop(labels=[1050,4557,12331],axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Unnamed: 0, Unnamed: 0.1, task, clip_task, cluster_type, index, Name, Gender, Age, Years, Age_id, Name_id]\n",
       "Index: []"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Unnamed: 0</th>\n      <th>Unnamed: 0.1</th>\n      <th>task</th>\n      <th>clip_task</th>\n      <th>cluster_type</th>\n      <th>index</th>\n      <th>Name</th>\n      <th>Gender</th>\n      <th>Age</th>\n      <th>Years</th>\n      <th>Age_id</th>\n      <th>Name_id</th>\n    </tr>\n  </thead>\n  <tbody>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "data[data.isnull().T.any()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0    0.578325\n4    0.060906\n2    0.057929\n5    0.051975\n1    0.048596\n3    0.046263\n9    0.041999\n7    0.041677\n8    0.038378\n6    0.033953\nName: cluster_type, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#查看各任务类型数量与占比\n",
    "print(data['cluster_type'].value_counts()/len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#看来第0类的任务占了半壁江山"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"数据处理 \n",
    "\n",
    "这里的clip_task是已经分词过的任务，所以不需要再进行分词操作\n",
    "\n",
    "我们使用sklearn的train_test_split来拆分训练集与测试集任务样本,基于类型分布比例来拆分样本。\n",
    "注意，我们这里进行任务分类模型的训练，输入特征样本为 clip_task,标签值为 cluster_type\n",
    "\"\"\"\n",
    "X_train,X_validation,Y_train,Y_validation =train_test_split(data[['clip_task']],data[['cluster_type']],test_size=0.2,stratify=data.cluster_type)  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0    0.578296\n4    0.060847\n2    0.057930\n5    0.051996\n1    0.048577\n3    0.046264\n9    0.042040\n7    0.041637\n8    0.038419\n6    0.033994\nName: cluster_type, dtype: float64\n\n0    0.578439\n4    0.061142\n2    0.057924\n5    0.051891\n1    0.048673\n3    0.046259\n9    0.041834\n7    0.041834\n8    0.038214\n6    0.033789\nName: cluster_type, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#验证一下拆分后的训练集与测试集分布比例是否一致\n",
    "dyt=Y_train.cluster_type.value_counts()/len(Y_train)\n",
    "dyv=Y_validation.cluster_type.value_counts()/len(Y_validation)\n",
    "print(dyt)\n",
    "print()\n",
    "print(dyv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#这里我们使用中文文学预训练的词向量模型,模型比较大，需要读取一会儿，我机器不好，用了大概3分钟。\n",
    "cn_model = KeyedVectors.load_word2vec_format(r'D:\\pythonOUT\\CGAI\\Models\\sgns.literature.word',\n",
    "                                            binary=False, unicode_errors='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "187959"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "len(cn_model.vocab)  #共18万词汇。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "300\n"
     ]
    }
   ],
   "source": [
    "test_vector=cn_model[cn_model.index2word[187298]]  #300长度的向量\n",
    "print(len(test_vector))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<bound method WordEmbeddingsKeyedVectors.word_vec of <gensim.models.keyedvectors.Word2VecKeyedVectors object at 0x000002776C3F0A90>>"
      ]
     },
     "metadata": {},
     "execution_count": 25
    }
   ],
   "source": [
    "cn_model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "28132"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "cn_model.vocab['动画'].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'MV 个人 视频 写真 个人 形象 专题 宣传片 拍摄 制作 剪辑 后期制作'"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "data.clip_task.values.tolist()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[100431, 5810, 41616, 41026, 14644, 749, 1555, 25223, 46948, 52666]"
      ]
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "source": [
    "#生成词索引列表\n",
    "x_train_tokens=[]\n",
    "for text in data.clip_task.values.tolist():\n",
    "    word_list=text.split(' ')\n",
    "    word_index_list=[]\n",
    "    for word in word_list:\n",
    "        try:\n",
    "            word_index = cn_model.vocab[word].index \n",
    "        except Exception as no_word_ERR:\n",
    "            word_index =0\n",
    "        word_index_list.append(word_index)\n",
    "    x_train_tokens.append(word_index_list)\n",
    "x_train_tokens[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "12429"
      ]
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "len(x_train_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "187298\n"
     ]
    }
   ],
   "source": [
    "max_index_list=[]\n",
    "for i in x_train_tokens:\n",
    "    per_max=max(i)\n",
    "    max_index_list.append(per_max)\n",
    "print(max(max_index_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'情侣卡'"
      ]
     },
     "metadata": {},
     "execution_count": 21
    }
   ],
   "source": [
    "cn_model.index2word[187298]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c"
   ]
  }
 ]
}